{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver \n",
    "import time \n",
    "import requests\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用edge进行数据爬取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "opts = webdriver.ChromeOptions()\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\n",
    "#opts.add_argument('blink-settings=imagesEnabled=false') #不加载图片, 提升速度\n",
    "\n",
    "driver_url = r\"C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedgedriver.exe\"\n",
    "\n",
    "driver = webdriver.Edge(executable_path=driver_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://www.cnki.net/')\n",
    "time.sleep(0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 查看是否登陆成功"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_id('Ecp_loginShowName1').get_attribute('innerHTML')\n",
    "driver.find_element_by_xpath('//*[@id=\"highSearch\"]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['CDwindow-07DF6B500508E37178B02502E582518B', 'CDwindow-093A51D39509A7E88067272F8E3D1BC6']\n"
     ]
    }
   ],
   "source": [
    "print (driver.window_handles)\n",
    "#切换窗口\n",
    "driver.switch_to.window(driver.window_handles[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('/html/body/div[3]/div[1]/div/ul[1]/li[1]/a').click()\n",
    "driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[5]/input').click()\n",
    "driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[6]/input').click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 开始搜索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('//*[@id=\"gradetxt\"]/dd[1]/div[2]/input').clear()#填入前先清除所有内容\n",
    "driver.find_element_by_xpath('//*[@id=\"gradetxt\"]/dd[1]/div[2]/input').send_keys(\"爬虫\")\n",
    "time.sleep(0.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[2]/input').click()\n",
    "time.sleep(0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_class_name('pagerTitleCell').get_attribute('innerHTML')\n",
    "time.sleep(0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21\n",
      "411\n",
      "<class 'int'>\n"
     ]
    }
   ],
   "source": [
    "results = driver.find_element_by_class_name('pagerTitleCell').find_element_by_tag_name('em').text\n",
    "all_re = int(results.replace(\",\",\"\"))\n",
    "num_results = int(all_re)\n",
    "num_pages = int(num_results/20) + 1\n",
    "print(num_pages)\n",
    "print(num_results)\n",
    "print(type(num_results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "element=driver.find_element_by_xpath('//div[@id=\"perPageDiv\"]/div/i')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "element=driver.find_element_by_xpath('//div[@id=\"perPageDiv\"]/ul/li[@data-val=\"50\"]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def downloader(title,href):\n",
    "    status_code = requests.get(href,headers=headers)\n",
    "    if status_code == 200:\n",
    "        contends = status_code.content\n",
    "        with open(f\"{title}.pdf\",\"wb\") as fp:\n",
    "            fp.wirte(contents)\n",
    "        print(f\"{title}\",\"成功下载\")\n",
    "    else:\n",
    "        print(\"无法下载\")\n",
    "def get_data(href):\n",
    "    DbCode = re.findall(\"DbCode=(.*?)&\",href)[0]\n",
    "    dbname = re.findall(\"dbname=(.*?)&\",href)[0]\n",
    "    filename = re.findall(\"filename=(.*?)&\",href)[0]\n",
    "    href = f\"https://kns.cnki.net/kcms/detail/detail.aspx?dbcode={DbCode}&dbname={dbname}&filename={filename}\"\n",
    "    return href\n",
    "\n",
    "def download(href):\n",
    "    page_source = requests.get(url=href,headers=headers).text\n",
    "    page_source_soup = etree.HTML(page_source)\n",
    "    download_url = page_source_soup.xpath('//li[@class=\"btn-dlpdf\"]/a/@href')\n",
    "    if download_url:\n",
    "        return urljoin(HOST,download_url[0])\n",
    "    else:\n",
    "        return \"没有下载链接\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from lxml import etree\n",
    "import re\n",
    "import requests\n",
    "from urllib.parse import urljoin\n",
    "HOST = \"https://kns.cnki.net/\"\n",
    "html = driver.page_source\n",
    "soup = etree.HTML(html)\n",
    "tr_list= soup.xpath('//div[@id=\"gridTable\"]/table/tbody/tr')\n",
    "headers = {\n",
    "    \"Cookie\":'Ecp_ClientId=3200615112602316607; cnkiUserKey=03369524-5653-0508-82c6-2fdbcdb21fe4; RsPerPage=20; _pk_ref=%5B%22%22%2C%22%22%2C1607047617%2C%22https%3A%2F%2Fwww.cnki.net%2F%22%5D; Ecp_ClientIp=202.116.81.140; UM_distinctid=178689abc586d3-0e68ae30691535-5771133-1fa400-178689abc597f5; Ecp_loginuserjf=15014134753; Ecp_session=1; ASP.NET_SessionId=gt3afeursglwft12mavvuvzf; SID_kns8=123111; SID_kns_new=kns123106; CurrSortFieldType=desc; SID_kcms=124103; SID_kxreader_new=011121; Hm_lvt_6e967eb120601ea41b9d312166416aa6=1621474773; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22179876d894d472-0e2dc24144c1fc-2363163-2073600-179876d894e760%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%7D%2C%22%24device_id%22%3A%22179876d894d472-0e2dc24144c1fc-2363163-2073600-179876d894e760%22%7D; Hm_lvt_ba7af201fc75865e9846f701ccb53e6b=1621474773; SID_kns=025123113; SID_klogin=125144; Hm_lpvt_6e967eb120601ea41b9d312166416aa6=1621475961; Hm_lpvt_ba7af201fc75865e9846f701ccb53e6b=1621475961; Ecp_loginuserbk=GDZSDX; knsLeftGroupSelectItem=null5%3B9%3B; CurrSortField=%e8%a2%ab%e5%bc%95%2f(%e8%a2%ab%e5%bc%95%e9%a2%91%e6%ac%a1%2c%27integer%27); _pk_ref=%5B%22%22%2C%22%22%2C1622444666%2C%22https%3A%2F%2Fwww.cnki.net%2F%22%5D; _pk_ses=*; _pk_id=90adc29b-dc70-48ff-8dcc-60e442d5a980.1619271339.5.1622444668.1622444666.; Ecp_LoginStuts={\"IsAutoLogin\":false,\"UserName\":\"GZ0513\",\"ShowName\":\"%e4%b8%ad%e5%b1%b1%e5%a4%a7%e5%ad%a6%e5%8d%97%e6%96%b9%e5%ad%a6%e9%99%a2\",\"UserType\":\"bk\",\"BUserName\":\"\",\"BShowName\":\"\",\"BUserType\":\"\",\"r\":\"4SVIml\"}; LID=WEEvREcwSlJHSldSdmVqM1BLVW9SQVdSTUtBeko5WUtLcE8xUldvbWhxRT0=$9A4hF_YAuvQ5obgVAqNKPCYcEjKensW4IQMovwHtwkF4VYPoHbKxJw!!; c_m_LinID=LinID=WEEvREcwSlJHSldSdmVqM1BLVW9SQVdSTUtBeko5WUtLcE8xUldvbWhxRT0=$9A4hF_YAuvQ5obgVAqNKPCYcEjKensW4IQMovwHtwkF4VYPoHbKxJw!!&ot=05/31/2021 15:24:33; c_m_expire=2021-05-31 15:24:33',\n",
    "    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36'\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_pages = 26 #根据显示页数填写\n",
    "\n",
    "\n",
    "title_list = []\n",
    "author_list = []\n",
    "href_list = []\n",
    "download_url_list = []\n",
    "\n",
    "def main():\n",
    "    now_page_count = 1\n",
    "    while now_page_count < num_pages:\n",
    "        html = driver.page_source\n",
    "        try:\n",
    "            soup = etree.HTML(html)\n",
    "            tr_list= soup.xpath('//div[@id=\"gridTable\"]/table/tbody/tr')\n",
    "            for tr in tr_list:\n",
    "                title = \"\".join(tr.xpath('td[@class=\"name\"]//text()')).strip().replace(\"\\n\",\"\").replace(\"                              网络首发\",\"\")\n",
    "                author = \",\".join(tr.xpath('td[@class=\"author\"]/a//text()')).strip().replace(\"\\n\",\"\")\n",
    "                href = tr.xpath('td[@class=\"name\"]/a/@href')[0]\n",
    "                href = get_data(href)\n",
    "                download_url = download(href)\n",
    "#                 print(download_url)\n",
    "                title_list.append(title)\n",
    "                author_list.append(author)\n",
    "                href_list.append(href)\n",
    "                download_url_list.append(download_url)\n",
    "            now_page_count = now_page_count + 1\n",
    "            driver.find_element_by_xpath('//*[@id=\"PageNext\"]').click()\n",
    "            time.sleep(0.5)\n",
    "        except:\n",
    "            time.sleep(10)# 解决500多遇到验证码问题\n",
    "    data = {\"标题\":title_list,\"作者\":author_list,\"链接\":href_list,\"下载链接\":download_url_list}\n",
    "    data = pd.DataFrame(data)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>作者</th>\n",
       "      <th>链接</th>\n",
       "      <th>下载链接</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>江南古镇旅游者地方依恋元素识别与维度构建——以乌镇为例</td>\n",
       "      <td>陆一帆,曲颖</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>数据爬取的刑法规制</td>\n",
       "      <td>孙杰</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>运用Dropout-LSTM模型的新冠肺炎趋势预测</td>\n",
       "      <td>王瑞,闫方,逯静,杨文艺</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>网络爬虫的演变及其合法性限定</td>\n",
       "      <td>苏青</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>打破“分娩者为母”的主导格局——新时代《民法典》妊娠代孕亲子关系的“开放包容式”认定</td>\n",
       "      <td>吴梓源</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560</th>\n",
       "      <td>国际关键词</td>\n",
       "      <td></td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>561</th>\n",
       "      <td>图像搜索引擎的体系结构及发展趋势研究</td>\n",
       "      <td>董文鸳,袁顺波</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562</th>\n",
       "      <td>知识解码(四)    生命进化知识解码(续)</td>\n",
       "      <td></td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>563</th>\n",
       "      <td>“聋”“盲”同源</td>\n",
       "      <td>游顺钊</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>564</th>\n",
       "      <td>关于哈萨克语中表示动物“叫”的动词</td>\n",
       "      <td>古丽夏提</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>565 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             标题            作者  \\\n",
       "0                   江南古镇旅游者地方依恋元素识别与维度构建——以乌镇为例        陆一帆,曲颖   \n",
       "1                                     数据爬取的刑法规制            孙杰   \n",
       "2                     运用Dropout-LSTM模型的新冠肺炎趋势预测  王瑞,闫方,逯静,杨文艺   \n",
       "3                                网络爬虫的演变及其合法性限定            苏青   \n",
       "4    打破“分娩者为母”的主导格局——新时代《民法典》妊娠代孕亲子关系的“开放包容式”认定           吴梓源   \n",
       "..                                          ...           ...   \n",
       "560                                       国际关键词                 \n",
       "561                          图像搜索引擎的体系结构及发展趋势研究       董文鸳,袁顺波   \n",
       "562                      知识解码(四)    生命进化知识解码(续)                 \n",
       "563                                    “聋”“盲”同源           游顺钊   \n",
       "564                           关于哈萨克语中表示动物“叫”的动词          古丽夏提   \n",
       "\n",
       "                                                    链接  \\\n",
       "0    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "1    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "2    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "3    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "4    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "..                                                 ...   \n",
       "560  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "561  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "562  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "563  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "564  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "\n",
       "                                                  下载链接  \n",
       "0    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "1    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "2    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "3    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "4    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "..                                                 ...  \n",
       "560  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "561  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "562  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "563  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "564  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "\n",
       "[565 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = main()\n",
    "display(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 存数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_excel('CNKI_数据.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
